package com.snr.userhistoryprocessor.urlprocessor;

import info.monitorenter.cpdetector.io.CodepageDetectorProxy;
import info.monitorenter.cpdetector.io.HTMLCodepageDetector;
import info.monitorenter.cpdetector.io.JChardetFacade;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.snr.userhistoryprocessor.utils.WebEncoding;

/**
 * TextExtractor鍔熻兘娴嬭瘯绫�
 */

public class UseDemo {
	private static CodepageDetectorProxy detector = CodepageDetectorProxy
			.getInstance();
	static {

		detector.add(new HTMLCodepageDetector(false));

		detector.add(JChardetFacade.getInstance());

	}

	public static void main(String args[]) throws IOException {
		String html = getHTML("http://stblog.baidu-tech.com/?p=1190");
		// System.out.println(html);
		String content = TextExtract.parse(html);
		System.out.println(";"+content);
	}

	public static String getHTML(String strurl) throws IOException {
		// TODO Auto-generated method stub
		String coding = WebEncoding.getCharset(strurl);
		StringBuffer sb = new StringBuffer();
		// 定义URL对象
		URL url = new URL(strurl);
		// 获取http连接对象
		HttpURLConnection urlConnection = (HttpURLConnection) url
				.openConnection();
		String line=null;
		System.out.println("+"+coding);
		BufferedReader reader=new BufferedReader(new InputStreamReader(urlConnection.getInputStream(),coding));
        while((line=reader.readLine())!=null){
            sb.append(line+"\n");
        }
		return sb.toString();
	}

	public static String getHTML3(String strurl) throws IOException {
		StringBuffer sb = new StringBuffer();
		// 定义URL对象
		URL url = new URL(strurl);
		// 获取http连接对象
		HttpURLConnection urlConnection = (HttpURLConnection) url
				.openConnection();
		;
		//long now = System.currentTimeMillis();
		urlConnection.connect();
		
		//System.out.println(System.currentTimeMillis()-now);
		// 网页编码
		String strencoding = null;

		/**
		 * 首先根据header信息，判断页面编码
		 */
		// map存放的是header信息(url页面的头信息)
		Map<String, List<String>> map = urlConnection.getHeaderFields();
		Set<String> keys = map.keySet();
		Iterator<String> iterator = keys.iterator();

		// 遍历,查找字符编码
		String key = null;
		String tmp = null;
		while (iterator.hasNext()) {
			key = iterator.next();
			tmp = map.get(key).toString().toLowerCase();
			// 获取content-type charset
			if (key != null && key.equals("Content-Type")) {
				int m = tmp.indexOf("charset=");
				if (m != -1) {
					strencoding = tmp.substring(m + 8).replace("]", "");
				}
			}
		}
	//	System.out.println(strencoding);
		if (strencoding==null) {
			String line;
			try {
			//	now = System.currentTimeMillis();
				BufferedReader in = new BufferedReader(new InputStreamReader(
						url.openStream()));
				while ((line = in.readLine()) != null) {
					sb.append(line);
				}
				in.close();
				//System.out.println(sb);
				//System.out.println(System.currentTimeMillis()-now);
			} catch (Exception e) { // Report any errors that arise
				System.err.println(e);
				System.err
						.println("Usage:   java   HttpClient   <URL>   [<filename>]");
			}
			String htmlcode = sb.toString();
			// 解析html源码，取出<meta />区域，并取出charset
			//System.out.println(htmlcode);
			String strbegin = "<meta";
			String strend = ">";
			String strtmp;
			int begin = htmlcode.indexOf(strbegin);
			int end = -1;
			int inttmp;
			//now = System.currentTimeMillis();
			//System.out.println(now);
			while (begin > -1) {
				//System.out.println(begin);
				end = htmlcode.substring(begin).indexOf(strend);
				if (begin > -1 && end > -1) {
					strtmp = htmlcode.substring(begin, begin + end).toLowerCase();
					//System.out.println(strtmp);
					inttmp = strtmp.indexOf("charset");
					//System.out.println(inttmp);
					if (inttmp > -1) {
						strencoding = strtmp.substring(inttmp + 7, end)
								.replace("=", "").replace("/", "")
								.replace("\"", "").replace("\'", "")
								.replace(" ", "");
						//return strencoding;
						break;
					}
				}
				htmlcode = htmlcode.substring(end);
				System.out.println(htmlcode);
				begin = htmlcode.indexOf(strbegin);
				//System.out.println(begin);
			}
		//	System.out.println(System.currentTimeMillis()-now);
			System.out.println(strencoding);
			if (strbegin==null) {
				/**
				 * 分析字节得到网页编码
				 */
				strencoding = getFileEncoding(url);

				// 设置默认网页字符编码
				if (strencoding == null) {
					strencoding = "GBK";
				}

			}
		}
		
		System.out.println(URLEncoder.encode(sb.toString(), strencoding));
		return sb.toString();
	}

	private static String getCharsetFromMetaAndFile(URL url) {
		/**
		 * 通过解析meta得到网页编码
		 */
		// 获取网页源码(英文字符和数字不会乱码，所以可以得到正确<meta/>区域)
		String strencoding = null;
		StringBuffer sb = new StringBuffer();
		String line;
		try {
			BufferedReader in = new BufferedReader(new InputStreamReader(
					url.openStream()));
			while ((line = in.readLine()) != null) {
				sb.append(line);
			}
			in.close();
		} catch (Exception e) { // Report any errors that arise
			System.err.println(e);
			System.err
					.println("Usage:   java   HttpClient   <URL>   [<filename>]");
		}
		String htmlcode = sb.toString();
		// 解析html源码，取出<meta />区域，并取出charset
		String strbegin = "<meta";
		String strend = ">";
		String strtmp;
		int begin = htmlcode.indexOf(strbegin);
		int end = -1;
		int inttmp;
		while (begin > -1) {
			end = htmlcode.substring(begin).indexOf(strend);
			if (begin > -1 && end > -1) {
				strtmp = htmlcode.substring(begin, begin + end).toLowerCase();
				inttmp = strtmp.indexOf("charset");
				if (inttmp > -1) {
					strencoding = strtmp.substring(inttmp + 7, end)
							.replace("=", "").replace("/", "")
							.replace("\"", "").replace("\'", "")
							.replace(" ", "");
					return strencoding;
				}
			}
			htmlcode = htmlcode.substring(begin);
			begin = htmlcode.indexOf(strbegin);
		}

		/**
		 * 分析字节得到网页编码
		 */
		strencoding = getFileEncoding(url);

		// 设置默认网页字符编码
		if (strencoding == null) {
			strencoding = "GBK";
		}

		return strencoding;
		// TODO Auto-generated method stub
		
	}

	public static String getHTML2(String strURL) throws IOException {
		URL url = new URL(strURL);
		BufferedReader br = new BufferedReader(new InputStreamReader(
				url.openStream()));
		String s = "";
		StringBuilder sb = new StringBuilder("");
		while ((s = br.readLine()) != null) {
			sb.append(s + "\n");
		}
		return sb.toString();
	}
	public static String getFileEncoding(URL url) {

		java.nio.charset.Charset charset = null;
		try {

			charset = detector.detectCodepage(url);

		} catch (Exception e) {

			System.out.println(e.getClass() + "分析" + "编码失败");

		}

		if (charset != null)

			return charset.name();

		return null;

	}
}
